import csv
import json
import re
from lxml import etree
import requests
from playwright.sync_api import sync_playwright

"""
爬取去哪儿旅行
"""

# 城市列表
url = "https://touch.piao.qunar.com/touch/toNewCityList.htm"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
}

response = requests.get(url, headers=headers)
root = etree.HTML(response.text)
# 境内城市
cityItem = root.xpath('//div[@id="city-domestic"]/ul[@class="mp-list"]/li/a')

cityList = []
for item in cityItem:
    cityList.append(item.text)
# print(cityList)
# print(len(cityList))


# 获取详情页
def get_detail(sightId):
    url = "https://touch.piao.qunar.com/touch/detail_{}.html".format(sightId)
    response = requests.get(url, headers=headers)
    root = etree.HTML(response.text)

    address = ""
    try:
        address = root.xpath('//p[@class="mp-baseinfo-address-txt"]/text()')[0]
    except:
        pass

    return address


# 获取景点介绍
def get_desc(sightId):
    url = "https://m.flight.qunar.com/gw/f/ticket/touch/sightIntroduction?sightId={}&channel=touch".format(
        sightId
    )

    introduce = ""
    try:
        response = requests.get(url, headers=headers)
        introduce = response.json()["data"]["introduce"]
    except:
        pass
    return introduce


# 获取评论页
def get_comments(sightId):
    url = (
        "https://touch.piao.qunar.com/touch/queryCommentsAndTravelTips.json?type=mp&pageSize=10&fromType=SIGHT&pageNum=1&sightId="
        + sightId
    )

    comment_total = 0
    commentList = []

    try:
        response = requests.get(url, headers=headers)
        comment = response.json()["data"]

        comment_total = comment.get("total")  # 总评论数
        arr = comment.get("commentList")  # 评论列表

        for c in arr:
            imgs = c.get("imgs")

            tmp = []
            if imgs:
                for img in imgs:
                    small = img.get("small")
                    tmp.append(small)

            item = {
                "headImg": "https:" + c.get("headImg"),
                "author": c.get("author"),
                "score": c.get("score"),
                "date": c.get("date"),
                "cityName": c.get("cityName"),
                "content": c.get("content"),
                "imgs": tmp,
            }
            commentList.append(item)
    except:
        pass

    return comment_total, commentList


# 获取景点图片
def get_sight_imgs(page, sightId):
    url = "https://touch.piao.qunar.com/touch/getSightImgs.htm?sightId=" + sightId

    img_list = []
    try:
        page.goto(url, wait_until="networkidle")
        page.wait_for_function("() => typeof window.context !== 'undefined'")
        imgData = page.evaluate("() => window.context.data.data")

        tmp = []
        for img in imgData:
            tmp.append("https:" + img["small"])
        img_list = tmp[:8]
    except:
        pass
    return img_list


# Top20 的景点
# https://touch.piao.qunar.com/touch/weekHotSales.htm?cityName=北京

row = [
    "city",
    "title",
    "cover",
    "sightId",
    "price",
    "interestNum",
    "score",
    "detailUrl",
    "address",
    "introduce",
    "img_list",
    "comment_total",
    "commentList",
]
with open("./tempData.csv", "a", encoding="utf8", newline="") as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(row)

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()  # 只打开一个page页面

    start = False
    for city in cityList:
        url = "https://touch.piao.qunar.com/touch/weekHotSales.htm?cityName=" + city

        if city == "白沙":
            start = True
        if not start:
            continue  # 跳过白沙前面的城市

        # 等待页面和js对象加载完成
        try:
            page.goto(url, wait_until="networkidle")
            page.wait_for_function("() => typeof window.context !== 'undefined'")
            sightList = page.evaluate("() => window.context.data.data.sightList")

            for travel in sightList:
                title = travel.get("title")  # 景点名称
                cover = travel.get("imageUrl")  # 封面
                price = travel.get("qunarPrice")  # 价格
                interestNum = travel.get("interestNum")  # 感兴趣人数
                score = travel.get("score")  # 评分
                detailUrl = "https:" + travel.get("scheme")  # 详情地址
                sightId = re.findall(r"detail_(\d+)", detailUrl)[0]  # 详情id

                # 获取详情页
                address = get_detail(sightId)

                # 获取景点介绍
                introduce = get_desc(sightId)

                # 获取景点图片列表(只取8张)
                img_list = get_sight_imgs(page, sightId)
                img_list = json.dumps(img_list)

                # 获取评论页(总评论数、评论列表)
                comment_total, commentList = get_comments(sightId)
                commentList = json.dumps(commentList)

                row = [
                    city,
                    title,
                    cover,
                    sightId,
                    price,
                    interestNum,
                    score,
                    detailUrl,
                    address,
                    introduce,
                    img_list,
                    comment_total,
                    commentList,
                ]
                with open(
                    "./tempData.csv", "a", encoding="utf8", newline=""
                ) as csvfile:
                    writer = csv.writer(csvfile)
                    writer.writerow(row)

                # break
            # break
        except:
            continue

    browser.close()

# 1、构建城市和省份的关系字典
# 2、